/*
 * Javlov - a Java toolkit for reinforcement learning with multi-agent support.
 * 
 * Copyright (c) 2009 Matthijs Snel
 * 
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
package net.javlov.policy;

import java.util.ArrayList;
import java.util.List;
import java.util.Random;

import net.javlov.Action;
import net.javlov.Policy;
import net.javlov.QFunction;
import net.javlov.State;
import net.javlov.util.ArrayUtil;

/**
 * Epsilon-greedy (e-greedy) policy. Chooses action with maximum Q-value with probability
 * 1 - e, and a (uniform) random action with probability e. This means that the greedy action
 * will be chosen with probability 1 - e + e/(nr of actions).
 *  
 * @author Matthijs Snel
 *
 */
public class EGreedyPolicy implements Policy {

	/**
	 * The Q-value function.
	 */
	private QFunction q;
	
	/**
	 * Epsilon.
	 */
	protected double e;
	
	/**
	 * List of allowed actions. Index of actions in the list should correspond to their
	 * ID.
	 */
	protected List<Action> actionPool;
	
	/**
	 * Random number generator.
	 */
	private Random rng;
	
	/**
	 * Creates an epsilon-greedy policy with the specified Q-function, epsilon, and pool
	 * of allowed actions.
	 * 
	 * @param q the Q-value function.
	 * @param epsilon action with maximum Q-value is chosen with probability
	 * 1 - e, and a (uniform) random action with probability e.
	 * @param actions list of allowed actions. Index of actions in the list should correspond to their
	 * ID.
	 */
	public EGreedyPolicy(QFunction q, double epsilon, List<? extends Action> actions) {
		setQFunction(q);
		setEpsilon(epsilon);
		actionPool = new ArrayList<Action>(actions);
		rng = new Random();
	}

	/**
	 * Chooses action with maximum Q-value (greedy action) with probability
	 * 1 - e, and a (uniformly distributed) random action with probability e. 
	 * This means that the greedy action will be chosen with probability
	 * {@code 1 - e + e/(nr of actions)}. If there is more than one greedy action, ties
	 * are broken randomly.
	 * 
	 * @param s the state based on which to choose the action. The Q-value function will
	 * be queried first to determine the Q-values of the actions for this state, after which
	 * the action will be determined by calling {@link #getAction(double[])}.
	 * @return an {@code Action} chosen according to the rule as specified above.
	 */
	@Override
	public <T> Action getAction(State<T> s) {
		return getAction( q.getValues(s) );
	}

	/**
	 * Chooses action with maximum Q-value (greedy action) with probability
	 * 1 - e, and a (uniformly distributed) random action with probability e. 
	 * This means that the greedy action will be chosen with probability
	 * {@code 1 - e + e/(nr of actions)}. If there is more than one greedy action, ties
	 * are broken randomly.
	 * 
	 * @param qvalues the q-values of the possible actions.
	 * @return an {@code Action} chosen according to the rule as specified above.
	 */
	@Override
	public Action getAction(double[] qvalues) {
		//choose greedy action, randomly break ties if there is more than one max action
		if ( Math.random() > e ) {
			//get indices of actions with max Q-value (returns 1 or more)
			int a[] = ArrayUtil.multimaxIndex(qvalues);
			return actionPool.get( a[rng.nextInt(a.length)] );
		}
		//choose random action
		return actionPool.get( rng.nextInt(qvalues.length) );
	}
	
	public void setQFunction(QFunction q) {
		this.q = q;
	}

	public QFunction getQFunction() {
		return q;
	}
	
	public double getEpsilon() {
		return e;
	}

	public void setEpsilon(double epsilon) {
		e = epsilon;
	}
}
